class Harvester
  require 'openssl/digest'
 gem 'twitter', '0.7.0'
 require 'digest/sha1' 

 #Todo refactor to somewhere else, e.g. the database
 @@FLICKR_API_KEY='c490c55e66e46c693833f3d9882a3e1e'
 @@FLICKR_SHARED_SECRET='0a4a91ba1549ceeb'
 @@SLIDESHARE_API_URL='http://www.slideshare.net/api/2'
 @@SLIDESHARE_API_KEY='oJUvtwhj'
 @@SLIDESHARE_SHARED_SECRET='4f5dEAWR'
 @@YOUTUBE_API_URL='http://gdata.youtube.com/feeds/api/videos'

 #google blog search test 
 @@GOOGLESEARCH_API_KEY='ABQIAAAAwPyazjbnigF3XARviQa33BRgzMovXYn9h7q0ClacHoK0cflX7BQKJzerY_s-ii1h23Rg7NuqLuHicg'
 @@GOOGLESEARCH_API_URL='http://ajax.googleapis.com/ajax/services/search/blogs'
 @@GOOGLESEARCH_HOST_IP='138.38.146.12'



 def self.request_harvest(tag)
   puts "Harvesting began on tag: #{tag.name}"
   err = ""
#   err = err + self.harvest_flickr(tag)
#   err = err + self.harvest_slideshare(tag)
#   err = err + self.harvest_tweet(tag)
#   err = err + self.harvest_youtube(tag)
   self.harvest_blog(tag)
   puts "Harvesting compleded: #{err}"
   if err != ("Flickr harvested. "+"Slideshare harvested. "+"Twitter harvested. "+"Youtube harvested. "+"Gblog harvested.")
     tag.update_attributes(:last_harvest => (Time.now.to_i-86400))
   end
   return err
 end


#Google blog search
def self.harvest_blog(tag)

  is_done = false
  page_no = 1
  start=0
  resultcount=8
  intialresult=nil
  while !is_done

                query_url = "#{@@GOOGLESEARCH_API_URL}?q=#{URI.escape(tag.name)}&v=1.0&userip=#{@@GOOGLESEARCH_HOST_IP}&rsz=#{resultcount}&start=#{start}&hl=en&key=#{@@GOOGLESEARCH_API_KEY}"
                retries = 2
                begin
                  json_data = Timeout::timeout(10){Net::HTTP.get_response(URI.parse(query_url)).body}
                  #puts json_data
                  hash_data = JSON.parse(json_data)

                  begin
                    results = hash_data['responseData']['results']
                  rescue
                    return
                  end

                 
                rescue Timeout::Error
                  if retries > 0
                    retries -= 1
                    retry
                  else
                    return "google blog timed out. "
                  end
                end
                if start == 0
                    intialresult=results[0]
                end

                start= start + resultcount
                i = 0
         
              if start != 0 || results[i] != intialresult
                      while results[i]!=nil


                                  result = results[i]
#                                  puts "*****************************"
#                                  puts "i=#{i}"
#                                  puts "title = #{result['title']}"
#                                  puts "blogurl = #{result['blogUrl']}"
#                                  puts "publishedDate = #{result['publishedDate']}"
#                                  puts "*****************************"
                                  @gblog = Gblog.new
                                  @gblog.title = result['title']
                                  @gblog.tags = tag.name
                                  @gblog.blogURL = result['blogUrl']
                                  @gblog.author = result['author']
                                  @gblog.publisheddate = result['publishedDate']

                                  @time = Time.at(result['publishedDate'].to_i)
                                  @gblog.upload_day = @time.day
                                  @gblog.upload_month = @time.mon
                                  @gblog.upload_year = @time.year

                                  @gblog.content = result['content']
                                  @gblog.tag_id = tag.id
                                  @gblog.save

                              i += 1
                        end
             else
                        #puts "Intial result=#{intialresult}"
                        is_done = true
             end
        
         
	   
	   page_no = page_no + 1
  end
  return "Gblog harvested. "
end

#called on a per user basis
def self.harvest_flickr(tag)
  flickr_url = "http://api.flickr.com/services/rest/"
  is_done = false
  page_no = 1
  while !is_done
  
    query_url = "#{flickr_url}?api_key=#{@@FLICKR_API_KEY}&method=flickr.photos.search&tags=#{URI.escape(tag.name)}&per_page=10&page=#{page_no}&extras=date_upload,date_taken,tags,views"
    retries = 2
    begin
      xml_data = Timeout::timeout(10){Net::HTTP.get_response(URI.parse(query_url)).body}
    rescue Timeout::Error
      if retries > 0
        retries -= 1
        retry
      else
        return "Flickr timed out. "
      end
    end
  
    doc = REXML::Document.new(xml_data)
    if(doc.root.attribute('stat').to_s=="ok")
      if doc.elements["rsp/photos"].attributes["pages"].to_i <= page_no
        is_done = true
      end
      doc.elements.each("rsp/photos/photo") { |element|  
        if not Flickr.find_by_photoid_and_tag_id(element.attributes["id"],tag.id)
          @flickr = Flickr.new
          @flickr.title = element.attributes["title"]
          @flickr.tags = element.attributes["tags"]
          @flickr.date_uploaded = element.attributes["dateupload"].to_i
  
          @time = Time.at(element.attributes["dateupload"].to_i)
          @flickr.upload_day = @time.day
  	   @flickr.upload_month = @time.mon
          @flickr.upload_year = @time.year
        
          @flickr.photoid = element.attributes["id"]
          @flickr.date_taken = element.attributes["datetaken"]
          @flickr.views = element.attributes["views"]
          @flickr.owner = element.attributes["owner"]
          @flickr.tag_id = tag.id
          @flickr.save
        end
      }
    else
      is_done = true
    end
    page_no = page_no + 1
  end
  return "Flickr harvested. "
end

def self.harvest_slideshare(tag)
  ts = Time.now.to_i
  hash = Digest::SHA1.hexdigest(@@SLIDESHARE_SHARED_SECRET.to_s+ts.to_s)

  is_done = false
  count = 0

  while !is_done
    tagencoded=CGI::escape(tag.name)
    #query_url = "#{@@SLIDESHARE_API_URL}/get_slideshows_by_tag?api_key=#{@@SLIDESHARE_API_KEY}&ts=#{ts}&hash=#{hash}&tag=#{tag.name}&limit=100&offset=#{count}&detailed=1"
    query_url = "#{@@SLIDESHARE_API_URL}/get_slideshows_by_tag?api_key=#{@@SLIDESHARE_API_KEY}&ts=#{ts}&hash=#{hash}&tag=#{tagencoded}&limit=100&offset=#{count}&detailed=1"
    count = count + 100
    retries = 2
    begin
     xml_data = Timeout::timeout(10){Net::HTTP.get_response(URI.parse(query_url)).body}
    rescue Timeout::Error
     if retries > 0
       retries -= 1
       retry
     else
       return "Slideshare timed out. "
     end
    end

    doc = REXML::Document.new(xml_data)
    if doc.elements["Tag/Count"] == nil || doc.elements["Tag/Count"].text.to_i <= count
      is_done = true
    end
    doc.elements.each("Tag/Slideshow") { |element|
      if !Slideshare.find_by_slideshow_id_and_tag_id(element.elements["ID"].text,tag.id)
        @slideshare = Slideshare.new
        @slideshare.slideshow_id = element.elements["ID"].text
        @slideshare.title = element.elements["Title"].text
        @slideshare.description = element.elements["Description"].text
        @slideshare.status = element.elements["Status"].text
        @slideshare.username = element.elements["Username"].text
        @slideshare.url = element.elements["URL"].text
        @slideshare.thumbnail_url = element.elements["ThumbnailURL"].text
        @slideshare.thumbnail_small_url = element.elements["ThumbnailSmallURL"].text
        @slideshare.embed = element.elements["Embed"].text
        @slideshare.created = Time.parse(element.elements['Created'].text.to_s)
        @time = @slideshare.created
        @slideshare.upload_day = @time.day
        @slideshare.upload_month = @time.month
        @slideshare.upload_year = @time.year
        @slideshare.language = element.elements["Language"].text
        @slideshare.format = element.elements["Format"].text
        temp = ""
        doc.elements.each("Tag/Slideshow/Tags/Tag") { |tag_element|
          temp << tag_element.text
          temp << ", "
        }
        @slideshare.tags = temp
        @slideshare.number_of_downloads = element.elements["NumDownloads"].text
        @slideshare.number_of_views = element.elements["NumViews"].text
        @slideshare.number_of_comments = element.elements["NumComments"].text
        @slideshare.number_of_favorites = element.elements["NumFavorites"].text
        @slideshare.number_of_slides = element.elements["NumSlides"].text
        temp = ""
        doc.elements.each("Tag/Slideshow/RelatedSlideshows/RelatedSlideshowID") { |related_element|
          temp << related_element.text
          temp << ", "
        }       
        @slideshare.related_slideshows = temp
        @slideshare.tag_id = tag.id
        @slideshare.save
      end
    }
  end
  return "Slideshare harvested. "
end
 
def self.harvest_tweet(tag)
  is_done = false
  page_no = 0
  begin
    while !is_done
      page_no = page_no + 1

      retries = 2
      begin
        twitter_search = Timeout::timeout(10){Twitter::Search.new.hashed(tag.name).page(page_no).per_page(100)}
          rescue Timeout::Error
        if retries > 0
          retries -= 1
          retry
        else
          return "Twitter timed out. "
        end
      end
  
      twitter_search.each do |r|
        if not Tweet.find_by_twitter_status_id_and_tag_id(r.id,tag.id)
          @tweet = Tweet.new
          @tweet.created_at = r.created_at
          @time = @tweet.created_at
          @tweet.upload_day = @time.day
          @tweet.upload_month = @time.month
          @tweet.upload_year = @time.year
          @tweet.twitter_status_id = r.id
          @tweet.text = r.text
          @tweet.source = r.source
          @tweet.in_reply_to_user_id = r.to_user_id
          @tweet.in_reply_to_screen_name = r.from_user
          @tweet.tag_id = tag.id
          @tweet.save
        end
      end  
    end
  rescue
    is_done = true
  end
  return "Twitter harvested. "
end
 
def self.harvest_youtube(tag)
  is_done = false
  count = 1
  tagencoded=CGI::escape(tag.name)
 
  while !is_done
    query_url = "#{@@YOUTUBE_API_URL}?q=#{tagencoded}&start-index=#{count}&max-results=50"
    count = count + 50

    retries = 2
    begin
      xml_data = Timeout::timeout(10){Net::HTTP.get_response(URI.parse(query_url)).body}
    rescue Timeout::Error
      if retries > 0
        retries -= 1
        retry
      else
        return "Slideshare timed out. "
      end
    end
    doc = REXML::Document.new(xml_data)
    if doc.elements["feed/openSearch:totalResults"].text.to_i <= count
      is_done = true
    end

    doc.elements.each("/feed/entry") { |element|
      if not Youtube.find_by_video_id_and_tag_id(element.elements["id"].text,tag.id)
        @youtube = Youtube.new
        @youtube.video_id = element.elements["id"].text
        @youtube.published = Time.parse(element.elements['published'].text.to_s)
        @youtube.updated = Time.parse(element.elements["updated"].text.to_s)
        @time = @youtube.updated
        @youtube.upload_day = @time.day
        @youtube.upload_month = @time.month
        @youtube.upload_year = @time.year
        @youtube.title = element.elements["title"].text
        @youtube.description = element.elements["content"].text
        @youtube.author = element.elements["author/name"].text
        @youtube.author_youtube_uri = element.elements["author/uri"].text

        if element.elements["gd:comments/gd:feedLink"] != nil
          @youtube.comment_feed = element.elements["gd:comments/gd:feedLink"].attributes["href"]
          @youtube.comment_count = element.elements["gd:comments/gd:feedLink"].attributes["countHint"]
        end
  
        if element.elements["media:group"] != nil
          @youtube.category = element.elements["media:group/media:category"].text
          @youtube.keywords = element.elements["media:group/media:keywords"].text
          @youtube.player_url = element.elements["media:group/media:player"].attributes["url"]
        end
          
        if element.elements["gd:rating"] != nil
          @youtube.number_of_raters = element.elements["gd:rating"].attributes["numRaters"]
          @youtube.average_rating = element.elements["gd:rating"].attributes["average"]
        end
  
        if element.elements["yt:statistics"] != nil        
          @youtube.favorite_count = element.elements["yt:statistics"].attributes["favoriteCount"]
          @youtube.view_count = element.elements["yt:statistics"].attributes["viewCount"]
        end
          
        @youtube.tag_id = tag.id
        @youtube.save
      end
    }
  end
  return "Youtube harvested. "
end
end
